#include "stdafx.h"
#include "SOPMI.h"
#include <math.h>
CSO_PMI::CSO_PMI(int dim,int bcnt,int pmiDist,const CString& workDir)
:m_dim(dim)
,m_bcnt(bcnt)
,m_pmiDist(pmiDist)
,m_workDir(workDir)
,m_alpha(0)
{}
bool CSO_PMI::init()
{
	readCount(m_workDir+CString("\\config\\featureCountListHybrid.dat"));
	loadClassCount(getCount(m_workDir+CString("\\config\\trainClasses.dat")));
	return true;
}
double CSO_PMI::GetAlpha()
{
	return m_alpha;
}
bool CSO_PMI::weighting(const CString &targetDir)
{
	traverse(targetDir);
	calculate();
	//countRecord();
	return true;
}
void CSO_PMI::countRecord()
{
	CStdioFile ofile;
	ofile.Open(m_workDir+"\\config\\soCountList.txt",CStdioFile::modeWrite|CStdioFile::modeCreate);
	rateWordMap::iterator iter=m_negList.begin();
	rateWordMap::iterator end =m_negList.end();
	for(;iter!=end;iter++)
	{
		CString str;
		std::vector<int> counts=m_countList[iter->second];
		int sum=0;
		for(int i=0;i<(int)counts.size();i++)
		{
			sum+=counts[i];
		}
		str.Format("%s %d\n",iter->second,sum);
		ofile.WriteString(str);
	}
	iter=m_posList.begin();
	end =m_posList.end();
	for(;iter!=end;iter++)
	{
		CString str;
		std::vector<int> counts=m_countList[iter->second];
		int sum=0;
		for(int i=0;i<(int)counts.size();i++)
		{
			sum+=counts[i];
		}
		str.Format("%s %d\n",iter->second,sum);
		ofile.WriteString(str);
	}
	ofile.Close();
}
bool CSO_PMI::readCount(const CString &Filename)
{
	CStdioFile iFile;
    if(!iFile.Open(Filename,CStdioFile::modeRead))
		return false;
    ULONGLONG len=iFile.GetLength();
    while(iFile.GetPosition()!=len)
    {
        CString str,name;
        int ipos,ineg;
        iFile.ReadString(str);
        sscanf(str,"%s %d %d",name,&ineg,&ipos);
        std::vector<int> count;
        count.push_back(ineg);
        count.push_back(ipos);
		CString x;
		x.Format("%s",name);
        m_countList.insert(wordCountPair(x,count));
    }
    iFile.Close();
	return true;
}
bool CSO_PMI::loadClassCount(std::vector<int> classCount)
{
	m_classCount=classCount;
	return true;
}
bool CSO_PMI::calculate()
{
	m_alpha=getAlpha();
	double max_v=0,min_v=0;
	PMIMap::iterator pmiIter=m_pmiMap.begin();
	PMIMap::iterator pmiEnd=m_pmiMap.end();
	for(;pmiIter!=pmiEnd;pmiIter++)
	{
		double sopmi=m_alpha;
		wordSingleCountMap countMap=pmiIter->second;
		sentiDict::iterator baseIter=m_baseDict.begin();
		sentiDict::iterator baseEnd=m_baseDict.end();
		for(;baseIter!=baseEnd;baseIter++)
		{
			int count=countMap.find(baseIter->first)->second;
			if(count==0)
				continue;
			if(baseIter->second==SNEG)
				sopmi-=log((double)count*1.0);
			else
				sopmi+=log((double)count*1.0);
		}
		if(sopmi>max_v)
			max_v=sopmi;
		if(sopmi<min_v)
			min_v=sopmi;
		if(m_totalDict[pmiIter->first]==SNEG)
			m_negList.insert(rateWordPair(sopmi,pmiIter->first));
		else
			m_posList.insert(rateWordPair(sopmi,pmiIter->first));
	}
	/*normalize(m_negList,max_v,min_v);
	normalize(m_negList,max_v,min_v);*/
	//min_v=m_alpha-max_v;
	m_alpha=(m_alpha-min_v)/(max_v-min_v)*2-1;
	writeDict(m_negList,m_workDir+CString("\\config\\neg.txt"),max_v,min_v);
	writeDict(m_posList,m_workDir+CString("\\config\\pos.txt"),max_v,min_v,true);
	return true;
}
bool CSO_PMI::fisher()
{

	wordCountMap::iterator  iter= m_countList.begin();
	wordCountMap::iterator  end= m_countList.end();
	for(;iter!=end;iter++)
    {
		double EP,EN,DP,DN,F;
		EP=((double)iter->second[SPOS]*1.0)/m_classCount[SPOS];
		EN=((double)iter->second[SNEG]*1.0)/m_classCount[SNEG];
		DP=( (1-EP)*(1-EP)*iter->second[SPOS] + EP*EP*(m_classCount[SPOS]-iter->second[SPOS]) ) / m_classCount[SPOS];
		DN=( (1-EN)*(1-EN)*iter->second[SNEG] + EN*EN*(m_classCount[SNEG]-iter->second[SNEG]) ) / m_classCount[SNEG];	  
		if(EP==EN)
			F=1.0;
		if(fabs(DP)<dZero||fabs(DN)<dZero)
			F=2.0;
		F=(EP-EN)*(EP-EN)/(DP+DN);
		m_FisherList.insert(rateWordPair(F,iter->first));  
    }
	return true;
}

bool CSO_PMI::traverse(const CString &targetDir)
{
	//
	sentiDict::iterator dictIter=m_baseDict.begin();
	sentiDict::iterator dictEnd=m_baseDict.end();
	wordSingleCountMap baseItem;
	for(;dictIter!=dictEnd;dictIter++)
	{
		baseItem.insert(wordSingleCountPair(dictIter->first,0));
	}
	//
	dictIter=m_totalDict.begin();
	dictEnd=m_totalDict.end();
	for(;dictIter!=dictEnd;dictIter++)
	{
		wordSingleCountMap item(baseItem);
		m_pmiMap.insert(PMIPair(dictIter->first,item));
	}
	//
	CFileFind cff;
	BOOL flag=cff.FindFile(targetDir+"\\*.*");
	while(flag)
	{		
		flag=cff.FindNextFile();
		if(cff.IsDirectory()&&!cff.IsDots())
		{
			CFileFind fFile;
			BOOL fileFlag=fFile.FindFile(cff.GetFilePath()+"\\*.*");
			while(fileFlag)
			{
				fileFlag=fFile.FindNextFile();
				if(!fFile.IsDirectory()&&!fFile.IsDots())
				{
					std::vector<CString> words;
					std::vector<int> poses;
					CStdioFile iFile;
					iFile.Open(fFile.GetFilePath().GetBuffer(),CStdioFile::modeRead);
					ULONGLONG len=iFile.GetLength();
					while(iFile.GetPosition()!=len)
					{
						CString str,name;
						int pos;
						iFile.ReadString(str);
						sscanf(str.GetBuffer(),"%s %d",name,&pos);
						CString x;
						x.Format("%s",name);
						words.push_back(x);
						poses.push_back(pos);
					}
					iFile.Close();
					sentiDict::iterator totalEnd=m_totalDict.end();
					sentiDict::iterator baseEnd=m_baseDict.end();
					for(int i=0;i<(int)words.size();i++)
					{
						if(m_baseDict.find(words[i])!=baseEnd)
						{
							for(int j=i+1;j<(int)words.size();j++)
							{
								if(poses[j]-poses[i]<m_pmiDist)
								{
									if(m_baseDict.find(words[j])!=baseEnd)
									{
										m_pmiMap[words[i]][words[j]]++;
										m_pmiMap[words[j]][words[i]]++;
									}
									else if(m_totalDict.find(words[j])!=totalEnd)
										m_pmiMap[words[j]][words[i]]++;
									
								}
								else break;
							}
						}
						else if(m_totalDict.find(words[i])!=totalEnd)
						{
							for(int j=i+1;j<(int)words.size();j++)
							{
								if(poses[j]-poses[i]<m_pmiDist)
								{
									if(m_baseDict.find(words[j])!=baseEnd)
										m_pmiMap[words[i]][words[j]]++;
										
								}
								else break;
							}
						}
					}

				}
			}
			fFile.Close();
		}
		
	}
	cff.Close();
	return true;
}
bool CSO_PMI::filter(const CString& dictDir)
{
	sentiDict bigDict;
	std::vector<CString> dictFiles;
	CFileFind find;
	BOOL flg=find.FindFile(dictDir+CString("\\*.*"));
	while(flg)
	{
		flg=find.FindNextFile();
		if(!find.IsDirectory()&&!find.IsDots())
			dictFiles.push_back(find.GetFilePath());
	}
	find.Close();
	readDict(dictFiles[SNEG],SNEG,bigDict);
	readDict(dictFiles[SPOS],SPOS,bigDict);
	int negCnt=0,posCnt=0;
	rateWordMap::reverse_iterator riter = m_FisherList.rbegin();
    rateWordMap::reverse_iterator rend  = m_FisherList.rend();
	
	for(;riter!=rend;riter++)
	{
		sentiDict::iterator sditer=bigDict.find(riter->second);
		if(sditer==bigDict.end())
			continue;		
		if(negCnt<m_bcnt&& sditer->second==SNEG)
		{
			m_baseDict.insert(sentiDictPair(sditer->first,sditer->second));
			negCnt++;
		}
		else if(posCnt<m_bcnt&& sditer->second==SPOS)
		{
			m_baseDict.insert(sentiDictPair(sditer->first,sditer->second));
			posCnt++;
		}
		m_totalDict.insert(sentiDictPair(sditer->first,sditer->second));
		if((int)m_totalDict.size()==m_dim)
			break;
	}


	CString str = m_workDir+"\\config\\baseWords.dat";
	CStdioFile oFile(str,CStdioFile::modeWrite|CStdioFile::modeCreate);
	for(int i=0;i<(int)m_classCount.size();i++)
	{
		CString header;
		header.Format("%d\n",i);
		oFile.WriteString(header);
		for(sentiDict::iterator iter=m_baseDict.begin();
		iter!=m_baseDict.end();iter++)
		{
			if(iter->second==i)
			{
				CString line;
				line.Format("%s\n",iter->first);
				oFile.WriteString(line);
			}
		}
	}
	oFile.Close();
	return false;
}
double CSO_PMI::getAlpha()
{
	double alpha=0;
	sentiDict::iterator iter=m_baseDict.begin();
	sentiDict::iterator end=m_baseDict.end();
	for(;iter!=end;iter++)
	{
		wordCountMap::iterator countIter=m_countList.find(iter->first);
		int total=countIter->second[SNEG]+countIter->second[SPOS];
		if(iter->second==SNEG)
			alpha+=log((double)total*1.0);
		else
			alpha-=log((double)total*1.0);
	}
	return alpha;
}
//void CSO_PMI::normalize(rateWordMap& wordList,double max,double min)
//{
//	rateWordMap::iterator iter=wordList.begin();
//	rateWordMap::iterator end =wordList.end();
//	for(;iter!=end;iter++)
//	{
//		iter->first=(iter->first-min)/(max-min)*2-1;
//	}
//}
bool readDict(CString &FileName,int classId,sentiDict &Dict)
{
   CStdioFile file;
   if(!file.Open(FileName,CStdioFile::modeRead))
       return false;
   CString buf,tmp;
   while(file.GetPosition()!=file.GetLength())
   {
       file.ReadString(buf);
       if(buf=="")
           continue;
       sscanf(buf,"%s",tmp);
       CString in;
       in.Format("%s",tmp);
       Dict.insert(sentiDictPair(in,classId));
   }
   file.Close();
   return true;
}
bool writeDict(rateWordMap &words,CString &filename,double max,double min,bool reverse)
{
    CStdioFile file;
    if(!file.Open(filename,CStdioFile::modeWrite|CStdioFile::modeCreate))
        return false;
	if(reverse)
	{
		rateWordMap::reverse_iterator riter = words.rbegin();
		rateWordMap::reverse_iterator rend	= words.rend();
		for(;riter!=rend;riter++)
		{
			double rate=riter->first;
			rate=(rate-min)/(max-min)*2-1;
			CString str;
			str.Format("%s %f\n",riter->second,rate);
			file.WriteString(str);
		}
	}
	else
	{
		rateWordMap::iterator iter = words.begin();
		rateWordMap::iterator end	= words.end();
		for(;iter!=end;iter++)
		{
			double rate=iter->first;
			rate=(rate-min)/(max-min)*2-1;
			CString str;
			str.Format("%s %f\n",iter->second,rate);
			file.WriteString(str);
		}
	}
	file.Close();
	return true;
}

std::vector<int> getCount(const CString& filename)
{
	std::vector<int> classCount;
	CStdioFile file;
    file.Open(filename,CStdioFile::modeRead);
	ULONGLONG len=file.GetLength();
	while(file.GetPosition()!=len)
	{
		CString tmp,name;
		int seq,count;
		file.ReadString(tmp);
		sscanf(tmp,"%d %s %d",&seq,name,&count);
		classCount.push_back(count);

	}
	file.Close();
    return classCount;
}
